package org.lionsoul.websnail;

import org.lionsoul.websnail.store.HashMapVisited;
import org.lionsoul.websnail.store.LinkedListTodo;
import org.lionsoul.websnail.task.DefaultTaskFactory;
import org.lionsoul.websnail.task.Task;
import org.lionsoul.websnail.worker.SpiderWorker;
import org.lionsoul.websnail.worker.Worker;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.locks.ReentrantLock;

/**
 * Spider main entrance class
 *
 * <p>handler the basic configuration of the current spider. offer the main controlling interface of
 * the spider
 *
 * @author chenxin<chenxin619315 @ gmail.com>
 */
public class Spider
{
    public static final int SPIDER_READY = 0; // ready to start
    public static final int SPIDER_RUNNING = 1;
    public static final int SPIDER_PAUSE = 2;
    public static final int SPIDER_STOPPED = 3;

    private final SpiderOptions options;

    private final Object LOCK = new Object(); // spider lock

    // spider running state
    private volatile int state = SPIDER_READY;

    // seed urls
    private final List<String> seedUrls;

    /**
     * spider worker to execute the page task
     */
    private final Worker worker;

    private final ReentrantLock lock;

    /**
     * create a new default spider
     *
     * @return Spider
     */
    public static Spider buildDefault()
    {
        return new Spider(null, null);
    }

    public Spider(SpiderOptions options, Worker worker)
    {
        // init spider options
        if (null == options) {
            this.options = SpiderOptions.custom().build();
        } else {
            // 初始化一些必传的参数值
            if (null == options.getTodoHandler()) {
                options.setTodoHandler(new LinkedListTodo());
            }
            if (null == options.getVisitedHandler()) {
                options.setVisitedHandler(new HashMapVisited());
            }
            if (null == options.getTaskFactory()) {
                options.setTaskFactory(new DefaultTaskFactory());
            }
            // 初始化 downloader
            this.options = options;
        }
        // init spider worker
        if (null == worker) {
            this.worker =
                    SpiderWorker.custom()
                            .setCoreThreadSize(100)
                            .setMaxThreadSize(300)
                            .setThreadAliveSecond(6)
                            .setThreadPoolQueue(new ArrayBlockingQueue<>(5000))
                            .setTaskQueue(new LinkedBlockingQueue<>(10000))
                            .build();
        } else {
            this.worker = worker;
        }

        this.seedUrls = new ArrayList<>(6);
        this.lock = new ReentrantLock();
    }

    public Spider(SpiderOptions options)
    {
        this(options, null);
    }

    public int getState()
    {
        return state;
    }

    public Spider setState(int state)
    {
        this.state = state;
        return this;
    }

    public SpiderOptions getOptions()
    {
        return options;
    }

    /**
     * create a specified kind of Task as for the task work detail depends on the task factory
     */
    public Task createTask(String url)
    {
        return options.getTaskFactory().createTask(this, url);
    }

    // add a seed url
    public Spider addSeedUrl(String url)
    {
        seedUrls.add(url);
        return this;
    }

    public Spider addSeedUrl(String[] urls)
    {
        seedUrls.addAll(Arrays.asList(urls));
        return this;
    }

    public Spider addSeedUrl(List<String> urls)
    {
        seedUrls.addAll(urls);
        return this;
    }

    public void pushRequestUrl(String link)
    {
        pushRequestUrl(link, false);
    }

    // 添加 URL 到 todolist 队列
    public void pushRequestUrl(String link, boolean force)
    {
        lock.lock();
        if (!force && options.getVisitedHandler().contains(link)) {
            lock.unlock();
            return;
        }

        // forced to add
        options.getVisitedHandler().add(link);
        options.getTodoHandler().add(link);
        lock.unlock();
    }

    public void pushRequestUrl(String[] urls)
    {
        for (String url : urls) {
            pushRequestUrl(url, false);
        }
    }

    public void pushRequestUrl(List<String> urls)
    {
        for (String url : urls) {
            pushRequestUrl(url, false);
        }
    }

    public Worker getWorker()
    {
        return worker;
    }

    /**
     * start the spider. <br>
     *
     * <p>1. start the worker first 2. keep fetching url from todo, then create a page task and send
     * it to the worker
     */
    public void start()
    {
        // 1. push the request for  the seed urls
        pushRequestUrl(seedUrls);

        // 2. start the worker
        worker.start();
        if (null != options.getSpiderListener()) {
            options.getSpiderListener().onStartup(this);
        }
        setState(SPIDER_RUNNING);

        // 3. keep sending task the the spider worker
        String url;
        while (true) {
            // check the running state and listen for shutdown
            // clear all resource and spider shutdown callback invoke
            if (state == SPIDER_STOPPED) {
                worker.shutdown();
                options.getTodoHandler().close();
                options.getVisitedHandler().close();

                if (null != options.getSpiderListener()) {
                    options.getSpiderListener().onShutdown(this);
                }

                break;
            }

            try {
                // check the running state and listen for pause
                if (state == SPIDER_PAUSE) {
                    synchronized (LOCK) {
                        if (null != options.getSpiderListener()) {
                            options.getSpiderListener().onPause(this);
                        }
                        LOCK.wait();
                    }
                }

                // 这里强制错开任务并发，减少爬虫被封杀的风险
                if (options.getTaskInterval() > 0 || worker.getLeftTask() > 10) {
                    Thread.sleep(options.getTaskInterval());
                }

                // get an url from the todo list
                url = options.getTodoHandler().shift();

                // Check if there is more url in the todo-list:
                // 1. no available url in the todo
                //
                // Or check if the spider's work is all accomplished:
                // 1. no available url in the todo
                // 2. no active working thread in the worker
                // 3. 连续 5 次没有拿到 todo-list 数据
                if (url == null) {
                    if (worker.getLeftTask() == 0 && worker.getWorkingThread() == 0) {
                        // 所有任务都完成了，进行回调
                        if (null != options.getSpiderListener()) {
                            options.getSpiderListener().onFinish(this);
                        }
                        break;
                    }

                    // sleep for a while, to wait for the incoming task url
                    Thread.sleep(options.getWorkerBusySleepTime());
                    continue;
                }

                // 创建爬虫任务
                Task task = createTask(url);
                // 尝试将任务添加到爬虫 Worker
                // 如果添加失败，可能是 Worker 任务队列已满，
                // 当前线程休眠一段时间再尝试添加，直到添加成功
                while (!worker.addTask(task)) {
                    Thread.sleep(options.getWorkerBusySleepTime());
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }

        // quit the spider
        shutdown();
    }

    // 暂停爬虫 worker，挂起任务
    public void pause()
    {
        if (state == SPIDER_PAUSE) {
            return;
        }
        setState(SPIDER_PAUSE);
        worker.pause();
    }

    // 恢复爬虫 Worker，继续任务
    public void resume()
    {
        if (state == SPIDER_RUNNING) return;
        synchronized (LOCK) {
            setState(SPIDER_RUNNING);
            LOCK.notify();
            if (null != options.getSpiderListener()) {
                options.getSpiderListener().onResume(this);
            }
        }

        // resume the worker
        worker.resume();
    }

    // 关闭爬虫，释放资源
    public void shutdown()
    {
        setState(Spider.SPIDER_STOPPED);
        worker.shutdown();
        options.getTodoHandler().close();
        options.getVisitedHandler().close();
        options.getDownloader().shutdown();
    }
}
